#/bin/sh
#set -x
SPLIT_SIZE=1024m #分包大小,支持的单位有k,m,g
SUB_PROCESS_NUM=20 #允许传输的并发的最大数量
SPEED_REFRESH_PEROID=2
FORCE=0

show_help()
{
    RET=$?
cat<<EOF
folder/file transfer tools V1.0
Usage:
`basename $0` [options] <source> <dest_username>@<dest_ip>:<dest_folder>
Details:
  source               the file/folder which need to be transfered
  dest_username        the dest host login username
  dest_ip              the dest host ip address
  dest_folder          the folder which the current host folder would place
  --split_size=value   the split size, the unit can be k,m,g. default value was ${SPLIT_SIZE}
  -f --force           force to clean local files and remote files
Example:
  `basename $0` /home/test user@10.10.1.1:/home/data 
  `basename $0` test2 user@10.10.1.1:/home/data 
EOF
    exit $RET
}

while test $# -gt 0; do
    #echo "$1"
    case "$1" in
    --split_size*)
        SPLIT_SIZE=$(echo "$1"|awk '{print substr($0,14)}')
        shift
        ;;
    --max_process=*)
        SUB_PROCESS_NUM=$(echo "$1"|awk '{print substr($0,16)}')
        shift
        ;;
    -f|--force)
        FORCE=1
        shift
        ;;
    -d|-debug|--debug)
        set -x
        shift
        ;;
    -h|-help|--help)
        show_help
        ;;
    -*)
        echo "unrecognized option $1" >&2
        echo "`basename $0` --help for more information." >&2
        exit 0
        ;;
    *)
        break
        ;;
    esac
done

if [ $# -lt 2 ];then
    echo "ERR: The given params was fewer" >&2
    show_help
fi

SRC_DIR=$1 #打包的源目录
DEST_DIR=$(echo "$2"|awk -F"@" '{print $2}'|awk -F":" '{print $2}') #目标主机的目录
DEST_IP=$(echo "$2"|awk -F"@" '{print $2}'|awk -F":" '{print $1}') #目标主机的IP
DEST_USER=$(echo "$2"|awk -F"@" '{print $1}') #目标主机用户名
SPLIT_FLAG=0
sleep_time=10 #循环检测任务时间
NEED_TAR=0 #文件夹才需要tar, 是文件则直接split
MATCH_PATTERN=""
parent_pid=$$

if [ ! -e "$SRC_DIR" ]; then
    echo "ERR: The src folder/file[$SRC_DIR] was not existed" >&2
    show_help
fi

name=`basename $SRC_DIR`
path=`dirname $SRC_DIR`
[ "$path" = "." ] && path=`pwd`

if [ -d $SRC_DIR ];then
    NEED_TAR=1
    FINAL_FILE="${name}.tar.gz"
elif [ -f $SRC_DIR ];then
    FINAL_FILE="${name}"
fi
MATCH_PATTERN="${FINAL_FILE}[a-z]"

cat<<EOF
source folder/file   : $path/$name
destination username : $DEST_USER
destination ip       : $DEST_IP
destination folder   : $DEST_DIR
allowed max process  : $SUB_PROCESS_NUM
tar split size       : $SPLIT_SIZE
match pattern        : $MATCH_PATTERN
EOF


TRANSFER_RECORD_FILE=$path/${name}_transfer.record #已经传输的文件记录
TAR_PID_FILE=$path/${name}_tar.pid

expected_size=$(python -c "human_size='$SPLIT_SIZE';tbl={'k':1024,'m':1024**2,'g':1024**3};last_char=human_size[-1].lower();
if last_char in tbl.keys():
    print tbl[last_char]*int(human_size[:-1])
else:
    print int(human_size)")

#创建FIFO做并发控制
tmp_fifofile=`mktemp`
rm -f $tmp_fifofile
tmp_fifofile="$tmp_fifofile.fifo"
mkfifo $tmp_fifofile
exec 6<>$tmp_fifofile
rm -f $tmp_fifofile

for ((i=0;i<$SUB_PROCESS_NUM;i++));do
echo
done >&6

cd $path
pid=$(cat $TAR_PID_FILE 2>/dev/null)
[ -z "$pid" ] && pid="123456789" #set to an uvalid pid
if [ $FORCE -eq 1 ]; then
    kill -9 $pid  2>/dev/null
    pkill -9 rsync
    sleep 2
    rm -f ${MATCH_PATTERN}*
    ssh -t $DEST_USER@$DEST_IP ". /etc/profile 1>/dev/null 2>&1; cd $DEST_DIR; rm -f ${MATCH_PATTERN}* .${MATCH_PATTERN}* 2>/dev/null "
fi

if kill -0 "$pid" 2>/dev/null ;then
    #the process was still alive
    echo "The tar process id[$pid] was alive"
else
    tar_split_files_count=`ls ${MATCH_PATTERN}* 2>/dev/null|wc -l`
    if [ $tar_split_files_count -lt 1 ];then
        SPLIT_FLAG=1
    fi

    if [ $SPLIT_FLAG -eq 1 ]; then
        rm -f $TRANSFER_RECORD_FILE
        if [ $NEED_TAR -eq 1 ];then
            tar czf - --exclude="$name/out*" $name |split -b $SPLIT_SIZE - $name.tar.gz &
            pid=$!
            echo $pid > $TAR_PID_FILE
            echo "The tar process id was $pid"
        else
            cat $name |split -b $SPLIT_SIZE - $name &
            pid=$!
            echo $pid > $TAR_PID_FILE
            echo "The split process id was $pid"
        fi
    fi
fi

progress()
{
    echo "start show progress ..."
    last_size=0
    total_size=0
    transfered_size=0
    alive_connection=0
	last_timestamp=0
    #total_size = transfered_size && total_size > 0 && alive_connection = 0
    while [ 1 ] ;do
        if ! kill -0 $parent_pid 1>/dev/null 2>/dev/null ; then
            echo "the parent process[$parent_pid] already existed"
            echo "so exited..."
            break
        fi
    
        sshRet=`ssh -t $DEST_USER@$DEST_IP ". /etc/profile 1>/dev/null 2>&1; cd $DEST_DIR;  ls -la ${MATCH_PATTERN}* .${MATCH_PATTERN}* 2>/dev/null" 2>/dev/null|awk -F"\r" '{print $1'}`
        current_size=`echo "$sshRet"|awk 'BEGIN{sum=0}{sum+=$5}END{print int(sum/1024)}'`
        transfered_size=`echo $current_size|awk '{print int($1/1024)}'`
        total_size=`ls -lG ${MATCH_PATTERN}* 2>/dev/null|awk 'BEGIN{sum=0}{sum+=$4}END{print int(sum/1024/1024)}'`
		current_timestamp=$[$(date +%s%N)/1000000]
        speed=0
        if [ $last_size -ne 0 ]; then
            speed=`echo "$current_size $last_size"|awk -v t_start=$last_timestamp -v t_end=$current_timestamp '{print ($1-$2)*1000.0/(t_end-t_start) }'`
        fi
        #echo "ps -fe|grep "rsync -a"|egrep "$MATCH_PATTERN"|grep -v grep"
        rsync_info=`ps -fe|grep "rsync -a"|egrep "$MATCH_PATTERN"|grep -v grep`
        #echo "$rsync_info"
        alive_connection=`echo "$rsync_info"|grep rsync|wc -l`
        #echo "alive_connection: $alive_connection"
        status=""
        if kill -0 $pid 1>/dev/null 2>/dev/null ; then
            if [ $NEED_TAR -eq 1 ];then
                status="istaring"
            else
                status="spliting"
            fi
        else
            percent=`echo "$transfered_size $total_size"|awk '{ print $1*100.0/$2}'`
            status=`printf "%3.3f%%" $percent`
        fi      
        printf "\rSpeed: %6s KB/s, transfered: %6s MB, total : %6s MB , connections: %2d status: %8s " $speed $transfered_size $total_size $alive_connection $status
        if [ $total_size -eq $transfered_size ] && [ $total_size -ne 0 ] && [ $alive_connection -eq 0 ];then
            break
        fi
        sleep $SPEED_REFRESH_PEROID
        last_size=$current_size
		last_timestamp=$current_timestamp
    done
    echo ""
    echo "exit the progress ..."
}

progress &

echo "start ..."
exit_flag=0
while [ $exit_flag -ne 2 ]
do
    sleep $sleep_time
    if kill -0 $pid 1>/dev/null 2>/dev/null ; then
       #echo "The tar process[$pid] was alive"
       echo >/dev/null
    else
        echo "The tar process[$pid] already existed!"
        echo "The tar process[$pid] already existed!"
        echo "The tar process[$pid] already existed!"
        if [ $exit_flag -eq 1 ];then
            exit_flag=2
        else
            #need to scan the directory once, maybe some files new appear
            exit_flag=1
        fi
    fi

    for f in ${MATCH_PATTERN}*
    do
        #echo "$f"
        if [ "$f" = "${MATCH_PATTERN}*" ];then
            echo "No file found, waiting ..."
            break
        fi
        
        if cat $TRANSFER_RECORD_FILE 2>/dev/null|grep $f >/dev/null;then
            #the file already in transfter
            #echo "The file[$f] already transfter"
            continue
        fi

        file_size=`ls -l $f|awk '{print $6}'`
        #only the tar process alive need to judge file_size
        if [ $file_size -ne $expected_size ] && [ $exit_flag -eq 0 ];then
            #echo "The [$f] size was $file_size, expected size: $expected_size "
            continue
        fi
        
        #previous transfer was still running
        is_in_transfer_info=`ps -fe|grep "rsync -a"|grep $f|grep -v grep|grep -v $$`
        if [ -n "$is_in_transfer_info" ];then
            #echo "$is_in_transfer_info"
            #echo "The [$f] already in transfer ..."
            continue
        fi
        
        read -u6
        {
            #echo "transfter $f ... begin"
            max_retry=3
            i=0
            while [ $i -lt $max_retry ];do
                let "i++"
                rsync -aqz $f ${DEST_USER}@${DEST_IP}:${DEST_DIR} 
                if [ $? -eq 0 ];then
                    echo $f >> $TRANSFER_RECORD_FILE
                    sync
                    sleep 1
                    break
                else
                    if ! kill -0 $parent_pid 1>/dev/null 2>/dev/null ; then
                        break
                    fi
                    echo "transfter $f ... FAILED, retry ...$i"
                    sleep 1
                fi
            done
            echo >&6
        } &
    done
    #in_processing=`cat $TRANSFER_PIDS_FILE 2>/dev/null|wc -l`
    #echo "There was $in_processing in processing"
done
if [ $exit_flag -eq 0 ]; then
    echo "error! it didn't enter the cycle" 
else
    echo "waiting all process to finish..."
fi
wait
echo "login to remote host, to merge,uncompress tar files..."
if [ $NEED_TAR -eq 1 ];then
	ssh -t $DEST_USER@$DEST_IP ". /etc/profile 1>/dev/null 2>&1; cd $DEST_DIR; cat ${MATCH_PATTERN}* > $FINAL_FILE && tar -xf $FINAL_FILE && rm -f ${MATCH_PATTERN}*"
else
	ssh -t $DEST_USER@$DEST_IP ". /etc/profile 1>/dev/null 2>&1; cd $DEST_DIR; cat ${MATCH_PATTERN}* > $FINAL_FILE && rm -f ${MATCH_PATTERN}*"
fi
echo "finished"



